#!/usr/bin/env python
# coding:utf-8

#
# 將聚會手記作後製整理
#
# Author: Chun-Yu Lee (Mat) <matlinuxer2@gmail.com>
# Copyright: Chun-Yu Lee (Mat) <matlinuxer2@gmail.com>
# License: MIT
#

from xmlrpclib import ServerProxy
import common
from common import *
import re
import os
import subprocess
import tempfile

ROOT = os.path.dirname(os.path.abspath(__file__))
cache_dir = os.path.join(ROOT, "_cache")

#
# 程式參數
#
read_settings_from_file()
api_user = volatile_settings['wikidot_api_user']
api_key  = volatile_settings['wikidot_api_key']
site     = 'hackingthursday'
root_url = 'http://www.hackingthursday.org'
proxy    = ServerProxy('https://' + api_user + ':' + api_key + '@www.wikidot.com/xml-rpc-api.php')
googledoc_user = volatile_settings['googledoc_email']
googledoc_pass = volatile_settings['googledoc_password']
spreadsheet    = volatile_settings['googledoc_spreadsheet']
worksheet      = volatile_settings['googledoc_worksheet']
dryrun       = volatile_settings['googledoc_dryrun']

#
# 程式資料結構
#
collection    = []
collection2   = []
collection3   = []
collection4   = []
user_mapping  = []
authors_cnt   = [] 

#
# 將頁面資料從 wikidot 下載到本地端
#
def pull_and_cache_pages(site, pages):
    while pages.__len__() > 0:
        # 一次取10個
        page_ary = pages[0:10]
        meta_ary = proxy.pages.get_meta({"site": site, "pages": page_ary})

        for page in page_ary:
            page_time = meta_ary[page]['updated_at']
            page_file = os.path.join(cache_dir, page + '@' + page_time)

            if os.access(cache_dir, os.R_OK | os.W_OK) is False:
                os.mkdir(cache_dir)

            if os.access(page_file, os.R_OK) is False:
                exist_pages = os.listdir(cache_dir)
                for p in exist_pages:
                    if p[:page.__len__()] == page:
                        p_path = os.path.join(cache_dir, p)
                        print "Remove duplicate: ", p
                        os.remove(p_path)

                print "Fetching page: ", page, " ..."
                the_page = proxy.pages.get_one({"site": site, "page": page})
                content = the_page["content"].encode('utf8')
                f = open(page_file, 'w')
                f.write(content)
                f.close()

        # 跳下一輪
        pages = pages[10:]


def get_page_content(page):
    result = ""

    exist_pages = os.listdir(cache_dir)
    for p in exist_pages:
        if p[:page.__len__()+1] == page+"@":
            p_path = os.path.join(cache_dir, p)
            result = file2string(p_path)

    return result

def list_headings(page):
    result = [];

    exist_pages = os.listdir(cache_dir)
    for p in exist_pages:
        if p[:page.__len__()] == page:
            p_path = os.path.join(cache_dir, p)
            cmd = "php parse_wikidot_heading.php --file='%s'" % p_path
            out = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE ).communicate()[0]
            result = out.strip().split('\n')

    return result


def append_data_per_author(author, date, data):
    global collection

    collection += [(author, date, data)]

def collect_unmapped_headding( heading, page ):
    global collection2

    collection2 += [(page, heading)]

def collect_notify( url_name ):
    global collection3

    collection3 += [ url_name ]

def collect_unnotify( url_name ):
    global collection4

    collection4 += [ url_name ]


def normalize_alias( alias ):
    result = alias
    result = result.replace(':'  , '')
    result = result.replace(','  , '')
    result = result.replace('.'  , '')
    result = result.replace('-'  , '')
    result = result.replace('\'' , '')
    result = result.replace('"'  , '')
    result = result.replace('_'  , '')
    result = result.replace(' '  , '')
    result = result.lower() 
    result = result.strip()

    return result

def find_author_by_heading( heading ):
    global user_mapping

    result = None 

    alias = normalize_alias( heading )

    for row in user_mapping:
        aliases = row['alias']
        for a in aliases:
                a2 = normalize_alias( a )
                if alias == a2:
                        result = row

    return result

def find_author_by_name( url_name ):
    global user_mapping

    result = None 

    for row in user_mapping:
        if url_name == row['url_name']:
                result = row

    return result

def partition_page(page):
    content   = get_page_content(page)
    headings  = list_headings(page)
    
    pos_ary      = []
    parts        = []
    content_rows = content.split("\n")

    for i in range(0, content_rows.__len__()):
        line = content_rows[i]
        #print i,": ", line
        result = re.findall('^\+ (.*)\s*$', line)
        if result.__len__() > 0:
            heading = result[0]
            if heading in headings:
                pos_ary += [(heading, i)]

    for i in range(0, pos_ary.__len__()):
        data = []
        if i + 1 == pos_ary.__len__():  # 最後一個
            for j in range(pos_ary[i][1] + 1, content_rows.__len__()):
                data += [content_rows[j]]
        else:
            for j in range(pos_ary[i][1] + 1, pos_ary[i + 1][1]):
                data += [content_rows[j]]

        heading = pos_ary[i][0]
        parts += [( heading, data )]

    return parts

def parse_data_from_page(page):
    parts = partition_page( page )

    for part in parts:
        heading_str = part[0]
        data        = part[1]

        heading_ary = heading_str.split(',')
        for heading in heading_ary:
            heading = heading.strip()

            # 如果該段是作者，則加入資料收集        
            author_data = find_author_by_heading( heading )
            if author_data != None:
                author = author_data['url_name']
                print "mapped  :: %s/%s :: %s => %s " % ( root_url, page, heading, author )
                append_data_per_author(author, page, data)
            else:
                print "unmapped:: %s/%s :: %s " % ( root_url, page, heading )
                collect_unmapped_headding( heading, page ) 


def list_authors():
    result = []
    for item in user_mapping:
        name = item['url_name']
        if name in result:
            continue
     
        if name.strip() == "":
            continue
        
        result += [name]

    return result


def make_index_of_authors():
    result = """
* [# H4ckers]
"""
    authors = list_authors()
    authors2 = []
    for au in authors:
	if get_author_cnt( au.lower() ) > 0:
		authors2.append( au )
    authors = authors2

    #en_char = "abcdefghijklmnopqrstuvwxyz"
    data_dict = {
        "abc": [],
        "def": [],
        "ghi": [],
        "jkl": [],
        "mno": [],
        "pqr": [],
        "stu": [],
        "vwx": [],
        "yz": [],
    }
    data_dict_nonen = []

    en_keys = data_dict.keys()
    en_keys.sort()  # 這裡作一次排序，以避免字母順序錯誤

    for author in authors:
        isNonEn = True
        for key in en_keys:
            if author[0].lower() in key and not author in data_dict[key]:
                    data_dict[key] += [author]
                    isNonEn = False
        if isNonEn:
            data_dict_nonen += [author]

    for key in en_keys:
        result += """ * [# 字母%s-%s]
""" % (key[:1].upper(), key[-1:].upper())
        items = data_dict[key]
        items.sort()  # 這裡作一次排序
        for item in items:

            # 這裡將作者名稱再轉成更個人化一些
            author_data = find_author_by_name( item )
            rel_name = author_data['rel_name']

            if item.lower() == rel_name.lower() or rel_name.strip() == "":
		    name = "%s" % ( item )
            else:
		    name = "%s (%s)" % ( rel_name, item )

            result += "  * [[[user:" + item + "|" + name + "]]]\n"

    # 目前 wikidot 不支援中文條目
    result += """ * [# 非英文字母]
  * [[[user:non-en]]]
"""

    for item in data_dict_nonen:
        print "非英文項目: ", item

    return result


def process_page_per_author(author):
    page_content = """
[[toc]]
"""

    for cc in collection:
        name = cc[0]
        date = cc[1]
        rows = cc[2]
        if name.lower() == author.lower():
            page_content += "\n+ " + date + "\n"
            page_content += "來源: [[[" + date + "]]]\n"
            for row in rows:
                page_content += row + "\n"

    page_content = page_content.strip() # Normalize

    return page_content

def wikidot_list_date_pages( proxy, site ):
        ret_data = []

        pages = proxy.pages.select({'site': site})
        for page in pages:
            if page[0:2] == "20" and page[4] == "-" and page[7] == "-":
                ret_data += [page]

        return ret_data
        
def wikidot_list_user_pages( proxy, site ):
        ret_data = []

        pages = proxy.pages.select({'site': site})
        for page in pages:
            if page[0:5] == "user:":
                ret_data += [page]

        return ret_data

def page_split_by_keyword( text, keyword ):
        pos = text.rfind(keyword)

        if pos >= 0:
            head = text[:pos]
            body = text[pos:]
        else:
            head = text
            body = ""

        return head, body

def get_author_cnt( rel_name ):
	global authors_cnt

	cnt = -1 
	for row in authors_cnt:
            if row[0] == rel_name:
		cnt = row[1]

	return cnt

#
# Main routine below
#

user_mapping_pre = fetch_googledoc_spreadsheet( googledoc_user, googledoc_pass, spreadsheet, worksheet )
user_mapping     = convert_spreadsheet_to_userdata( user_mapping_pre )

# 列出要處理的頁面
user_pages = wikidot_list_user_pages( proxy, site )
date_pages = wikidot_list_date_pages( proxy, site )
print "Fetching 聚會手記..."
pull_and_cache_pages(site, date_pages )  # 下載聚會手記相關的頁面
print "Fetching 個人頁面..."
pull_and_cache_pages(site, user_pages )  # 下載個人頁面相關的頁面

date_pages.sort(reverse=True)  # 將最新日期放在最前面
for page in date_pages:
    print "處理頁面: ", page
    parse_data_from_page(page)

# 
# 列出每個作者的統計資訊
# 
print "==== 列出每個作者的統計資訊 ============================"
authors = list_authors()
for author in authors:
    cnt = 0 
    for cc in collection:
        name = cc[0]
	if name == author:
		cnt += 1
    authors_cnt += [ ( author, cnt ) ]

authors_cnt.sort( key=lambda x: x[1], reverse=True )

for row in authors_cnt:
    print "%s  %s" % ( str(row[1]).rjust(4), row[0] ) 

#
# 更新個別作者的頁面
#
print "==== 更新個別作者的頁面 ========================="
authors = list_authors()
for author in authors:
    page_of_author = process_page_per_author(author)
    page = "user:" + author.lower()
    user_head, page_of_author_orig = page_split_by_keyword( get_page_content(page), "[[toc]]" )
    cnt = get_author_cnt( author.lower() )

    if page_of_author == page_of_author_orig or cnt <= 0:
        # 內容沒有變動，或無個人筆記的，跳過不更新
        continue

    # 顯示差異的部分
    tmp_old = tempfile.mktemp()
    tmp_new = tempfile.mktemp()
    string2file( page_of_author_orig, tmp_old )
    string2file( page_of_author, tmp_new )
    cmd = "diff -Naur %s %s" % ( tmp_old, tmp_new )
    out = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE ).communicate()[0]
    os.unlink( tmp_old )
    os.unlink( tmp_new )
    print "==== %s 新舊差異的部分 ===========" % author
    print out
    print "=================================="

    page_of_author = user_head + page_of_author # 加上保留的使用者編輯內容

    author_data = find_author_by_name( author )
    name   = author_data['rel_name']
    notify = author_data['notify']

    notify_enable  = False
    if notify.lower().strip() in [ "on", "v", "o", "yes", "y" ]:
	notify_enable = True

    title = name + "的聚會手記"

    print "更新個人頁面: %s/%s" % ( root_url, page )
    if dryrun != True:
        if notify_enable == True:
	    proxy.pages.save_one({'site': site, 'page': page, 'title': title, 'content': page_of_author})
            collect_notify( author )
	else:
            print "Notify 未開通: %s => %s/%s" % ( author, root_url, page )
            collect_unnotify( author )
	
    

#
# Calucate user statics
#
ary_local  = list_authors()
ary_remote = []
for user_page in user_pages:
	user = user_page[5:]
	ary_remote += [ user ]

ary_all = set( ary_local + ary_remote )

ary_local_gt_remote = []
ary_local_eq_remote = []
ary_local_lt_remote = []

for item in ary_all:
	if item in ary_local and item in ary_remote:
		ary_local_eq_remote += [ item ]
	elif item in ary_local and not item in ary_remote:
		ary_local_gt_remote += [ item ]
	elif not item in ary_local and item in ary_remote:
		ary_local_lt_remote += [ item ]
	else:
		print "例外: %s" % item 

print "== 本地有，遠端有 ======================"
for x in ary_local_eq_remote:
	print "%s/user:%s" % ( root_url, x )
print "== 本地有，遠端沒有 ===================="
for x in ary_local_gt_remote:
	print "%s/user:%s" % ( root_url, x )
print "== 本地沒有，遠端有 ===================="
for x in ary_local_lt_remote:
	print "%s/user:%s" % ( root_url, x )
print "========================================"


# 展開作者列表
page_of_index = make_index_of_authors()
print page_of_index

# 將作者列表塞到主選單中
nav_page = "nav:top"
keyword_beg = "\n* [# H4ckers]\n"
keyword_end = "\n  * [[[user:non-en]]]\n"

nav_top_page = proxy.pages.get_one({"site": site, "page": nav_page})
nav_top_content = nav_top_page["content"].encode('utf8')
k_start = nav_top_content.find(keyword_beg)
k_end = nav_top_content.find(keyword_end)

if k_start >= 0 and k_end >= 0 and k_end > k_start:
    k_end = k_end + keyword_end.__len__()

    new_nav_top_content = nav_top_content[0:k_start] + page_of_index + nav_top_content[k_end:]

    #print new_nav_top_content
    if dryrun != True:
	    proxy.pages.save_one({'site': site, 'page': nav_page, 'content': new_nav_top_content})


def send_notify_mail( author ):
    author_data = find_author_by_name( author )
    rel_name = author_data['rel_name']
    url_name = author_data['url_name']
    email    = author_data['email']

    Sender  = googledoc_user  # 值日生的 email
    Reciver = email
    Subject = "H4個人頁面更新通知"
    Link    = "%s/user:%s" % ( root_url, url_name )
    
    Html = """
<html>
    <head>
        <title>%s</title>
    </head>
    <body>
    <a href="%s">%s</a>
    <pre>
Hi %s 您好:

關於您在 H4 的個人手記有新的
內容嘍~(網址如上) 歡迎您有空
再來 H4 逛逛~!

備註: 
這是 H4 值日生們新推出的小功
能，bot 會將大家的筆記，依個
別作排序跟整理在 wiki 上。

%s

備註2:
個人頁面 "Table of Contents"
以上的部份是可以編輯的。歡迎
您編輯個人相關資訊及連結!! 
( Table of Contents 以下的部
份則是會由 bot 定期產生並覆
蓋 )

若您有任何問題或是建議，都歡
迎 feedback 給我們，謝謝!!
    </pre>
    </body>
</html>
"""  % ( Subject, Link, Link, rel_name, root_url+"h4note" )

    Txt = common.html2txt(Html)

    receivers = Reciver.split(',')
    for item in receivers:
        receiver = item.strip()
        common.send_gmail(Sender, receiver, Subject, Txt, Html, googledoc_user, googledoc_pass )

#
# 通知作者有頁面更新
#
print "==== 通知作者有頁面更新 =================================="
for author in collection3:
    author_data = find_author_by_name( author )
    rel_name = author_data['rel_name']
    url_name = author_data['url_name']
    email    = author_data['email']
    notify   = author_data['notify']
    notify_enable  = False

    if notify.lower().strip() in [ "on", "v", "o", "yes", "y" ]:
	notify_enable = True

    if notify_enable:
	    print 'To nofity: "%s" <%s> , %s/user:%s ' % ( rel_name, email, root_url, url_name )
	    if dryrun != True:
		send_notify_mail( author )
    else:
	    print '  skiping: "%s" <%s> , %s/user:%s ' % ( rel_name, email, root_url, url_name )

print "==== 未對應之作者    =================================="
for item in collection2:
	print "%s/%s '%s' " % ( root_url, item[0], item[1] )

print "==== 未開通之作者    =================================="
for item in collection4:
	print item
